From b2ad353b42620cc4bbda18fada7d0467fb0e737a Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 23 Feb 2012 18:19:00 -0500 Subject: [PATCH] core: Store "pending" objects explicitly, fix pull to use them Before we were creating randomly-named temporary files in repo/tmp when downloading via pull, but that means if the download process is interrupted, we have to redownload everything again. Let's still keep the concept of a "transaction" where files are stored in the repository as atomically as possible (i.e. we do a bunch of rename() calls), but now we also have an explicit "tmp/pending/objects" directory that contains named objects. This allows us to then skip redownloading things that are pending. --- src/libostree/ostree-repo.c | 250 ++++++++++++++++++----------------- src/libostree/ostree-repo.h | 13 +- src/ostree/ostree-pull.c | 257 +++++++++++++++++++++++------------- 3 files changed, 297 insertions(+), 223 deletions(-) diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index d7c3ff2b..eee4bdfa 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -56,6 +56,7 @@ typedef struct _OstreeRepoPrivate OstreeRepoPrivate; struct _OstreeRepoPrivate { GFile *repodir; GFile *tmp_dir; + GFile *pending_dir; GFile *local_heads_dir; GFile *remote_heads_dir; GFile *objects_dir; @@ -63,12 +64,11 @@ struct _OstreeRepoPrivate { gboolean inited; gboolean in_transaction; - GFile *transaction_dir; GKeyFile *config; OstreeRepoMode mode; - GHashTable *pending_transaction_tmpfiles; + GHashTable *pending_transaction; }; static void @@ -79,12 +79,12 @@ ostree_repo_finalize (GObject *object) g_clear_object (&priv->repodir); g_clear_object (&priv->tmp_dir); + g_clear_object (&priv->pending_dir); g_clear_object (&priv->local_heads_dir); g_clear_object (&priv->remote_heads_dir); g_clear_object (&priv->objects_dir); g_clear_object (&priv->config_file); - g_clear_object (&priv->transaction_dir); - g_hash_table_destroy (priv->pending_transaction_tmpfiles); + g_hash_table_destroy (priv->pending_transaction); if (priv->config) g_key_file_free (priv->config); @@ -149,6 +149,7 @@ ostree_repo_constructor (GType gtype, g_assert (priv->repodir != NULL); priv->tmp_dir = g_file_resolve_relative_path (priv->repodir, "tmp"); + priv->pending_dir = g_file_resolve_relative_path (priv->repodir, "tmp/pending"); priv->local_heads_dir = g_file_resolve_relative_path (priv->repodir, "refs/heads"); priv->remote_heads_dir = g_file_resolve_relative_path (priv->repodir, "refs/remotes"); @@ -184,9 +185,9 @@ ostree_repo_init (OstreeRepo *self) { OstreeRepoPrivate *priv = GET_PRIVATE (self); - priv->pending_transaction_tmpfiles = g_hash_table_new_full (g_str_hash, g_str_equal, - g_free, - g_free); + priv->pending_transaction = g_hash_table_new_full (g_str_hash, g_str_equal, + g_free, + NULL); } OstreeRepo* @@ -600,6 +601,9 @@ ostree_repo_check (OstreeRepo *self, GError **error) ot_gfile_get_path_cached (priv->objects_dir)); goto out; } + + if (!ot_gfile_ensure_directory (priv->pending_dir, FALSE, error)) + goto out; priv->config = g_key_file_new (); if (!g_key_file_load_from_file (priv->config, ot_gfile_get_path_cached (priv->config_file), 0, error)) @@ -700,35 +704,60 @@ create_checksum_and_objtype (const char *checksum, return g_strconcat (checksum, ".", ostree_object_type_to_string (objtype), NULL); } +static GFile * +get_pending_object_path (OstreeRepo *self, + const char *checksum, + OstreeObjectType objtype) +{ + OstreeRepoPrivate *priv = GET_PRIVATE (self); + char *relpath; + GFile *ret; + + relpath = ostree_get_relative_object_path (checksum, objtype); + ret = g_file_resolve_relative_path (priv->pending_dir, relpath); + g_free (relpath); + + return ret; +} + gboolean -ostree_repo_has_object (OstreeRepo *self, - OstreeObjectType objtype, - const char *checksum, - gboolean *out_have_object, - GCancellable *cancellable, - GError **error) +ostree_repo_find_object (OstreeRepo *self, + OstreeObjectType objtype, + const char *checksum, + GFile **out_stored_path, + GFile **out_pending_path, + GCancellable *cancellable, + GError **error) { gboolean ret = FALSE; - OstreeRepoPrivate *priv = GET_PRIVATE (self); - char *tmp_key = NULL; GFile *object_path = NULL; + struct stat stbuf; - tmp_key = create_checksum_and_objtype (checksum, objtype); + g_return_val_if_fail (out_stored_path, FALSE); + g_return_val_if_fail (out_pending_path, FALSE); - if (g_hash_table_lookup (priv->pending_transaction_tmpfiles, tmp_key)) + object_path = ostree_repo_get_object_path (self, checksum, objtype); + + *out_stored_path = NULL; + *out_pending_path = NULL; + if (lstat (ot_gfile_get_path_cached (object_path), &stbuf) == 0) { - *out_have_object = TRUE; + *out_stored_path = object_path; + object_path = NULL; } else { - object_path = ostree_repo_get_object_path (self, checksum, objtype); - - *out_have_object = g_file_query_exists (object_path, cancellable); + g_clear_object (&object_path); + object_path = get_pending_object_path (self, checksum, objtype); + if (lstat (ot_gfile_get_path_cached (object_path), &stbuf) == 0) + { + *out_pending_path = object_path; + object_path = NULL; + } } ret = TRUE; /* out: */ - g_free (tmp_key); g_clear_object (&object_path); return ret; } @@ -755,36 +784,53 @@ stage_object_impl (OstreeRepo *self, GCancellable *cancellable, GError **error); -static gboolean +static void insert_into_transaction (OstreeRepo *self, const char *checksum, - OstreeObjectType objtype, - GFile *tempfile_path, - GError **error) + OstreeObjectType objtype) { - gboolean ret = FALSE; OstreeRepoPrivate *priv = GET_PRIVATE (self); char *key; key = create_checksum_and_objtype (checksum, objtype); + /* Takes ownership */ + g_hash_table_replace (priv->pending_transaction, key, NULL); +} + +static gboolean +stage_tmpfile_trusted (OstreeRepo *self, + const char *checksum, + OstreeObjectType objtype, + GFile *tempfile_path, + GError **error) +{ + gboolean ret = FALSE; + GFile *pending_path = NULL; + GFile *checksum_dir = NULL; + + pending_path = get_pending_object_path (self, checksum, objtype); + checksum_dir = g_file_get_parent (pending_path); + + if (!ot_gfile_ensure_directory (checksum_dir, TRUE, error)) + goto out; - if (g_hash_table_lookup (priv->pending_transaction_tmpfiles, key)) + if (link (ot_gfile_get_path_cached (tempfile_path), ot_gfile_get_path_cached (pending_path)) < 0) { - if (unlink (ot_gfile_get_path_cached (tempfile_path)) < 0) + if (errno != EEXIST) { ot_util_set_error_from_errno (error, errno); goto out; } - g_free (key); - } - else - { - g_hash_table_insert (priv->pending_transaction_tmpfiles, - key, g_strdup (ot_gfile_get_basename_cached (tempfile_path))); } + insert_into_transaction (self, checksum, objtype); + + (void) unlink (ot_gfile_get_path_cached (tempfile_path)); + ret = TRUE; out: + g_clear_object (&pending_path); + g_clear_object (&checksum_dir); return ret; } @@ -816,7 +862,7 @@ impl_stage_archive_file_object_from_raw (OstreeRepo *self, g_variant_get_size (serialized), NULL); - if (!ostree_create_temp_file_from_input (priv->transaction_dir, + if (!ostree_create_temp_file_from_input (priv->tmp_dir, "archive-tmp-", NULL, NULL, NULL, mem, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META, @@ -826,7 +872,7 @@ impl_stage_archive_file_object_from_raw (OstreeRepo *self, goto out; temp_info = dup_file_info_owned_by_me (file_info); - if (!ostree_create_temp_file_from_input (priv->transaction_dir, + if (!ostree_create_temp_file_from_input (priv->tmp_dir, "archive-tmp-", NULL, temp_info, NULL, input, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT, @@ -861,12 +907,12 @@ impl_stage_archive_file_object_from_raw (OstreeRepo *self, else actual_checksum = g_checksum_get_string (ret_checksum); - if (!insert_into_transaction (self, actual_checksum, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT, - content_temp_file, error)) + if (!stage_tmpfile_trusted (self, actual_checksum, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT, + content_temp_file, error)) goto out; - if (!insert_into_transaction (self, actual_checksum, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META, - meta_temp_file, error)) + if (!stage_tmpfile_trusted (self, actual_checksum, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META, + meta_temp_file, error)) goto out; ret = TRUE; @@ -898,7 +944,8 @@ stage_object_impl (OstreeRepo *self, GChecksum *ret_checksum = NULL; GFileInfo *temp_info = NULL; GFile *temp_file = NULL; - gboolean already_exists; + GFile *stored_path = NULL; + GFile *pending_path = NULL; const char *actual_checksum; g_return_val_if_fail (priv->in_transaction, FALSE); @@ -910,29 +957,28 @@ stage_object_impl (OstreeRepo *self, if (expected_checksum) { - if (!ostree_repo_has_object (self, objtype, expected_checksum, &already_exists, cancellable, error)) + if (!ostree_repo_find_object (self, objtype, expected_checksum, &stored_path, &pending_path, + cancellable, error)) goto out; } - else - already_exists = FALSE; g_assert (objtype != OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT); g_assert (objtype != OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META); - if (objtype == OSTREE_OBJECT_TYPE_RAW_FILE) + if (stored_path == NULL && pending_path == NULL) { - g_assert (file_info != NULL); - if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR) - g_assert (input != NULL); - } - else if (OSTREE_OBJECT_TYPE_IS_META (objtype)) - { - g_assert (xattrs == NULL); - g_assert (input != NULL); - } + if (objtype == OSTREE_OBJECT_TYPE_RAW_FILE) + { + g_assert (file_info != NULL); + if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR) + g_assert (input != NULL); + } + else if (OSTREE_OBJECT_TYPE_IS_META (objtype)) + { + g_assert (xattrs == NULL); + g_assert (input != NULL); + } - if (!already_exists) - { if (objtype == OSTREE_OBJECT_TYPE_RAW_FILE && priv->mode == OSTREE_REPO_MODE_ARCHIVE) { if (!impl_stage_archive_file_object_from_raw (self, file_info, xattrs, input, @@ -943,7 +989,7 @@ stage_object_impl (OstreeRepo *self, } else { - if (!ostree_create_temp_file_from_input (priv->transaction_dir, + if (!ostree_create_temp_file_from_input (priv->tmp_dir, ostree_object_type_to_string (objtype), NULL, file_info, xattrs, input, objtype, @@ -967,13 +1013,23 @@ stage_object_impl (OstreeRepo *self, } } - if (!insert_into_transaction (self, actual_checksum, objtype, - temp_file, error)) + if (!stage_tmpfile_trusted (self, actual_checksum, objtype, + temp_file, error)) goto out; g_clear_object (&temp_file); } } - + else if (pending_path) + { + g_assert (expected_checksum); + insert_into_transaction (self, expected_checksum, objtype); + } + else + { + g_assert (stored_path != NULL); + /* Nothing to do */ + } + ret = TRUE; ot_transfer_out_value(out_checksum, &ret_checksum); out: @@ -981,6 +1037,8 @@ stage_object_impl (OstreeRepo *self, (void) unlink (ot_gfile_get_path_cached (temp_file)); g_clear_object (&temp_file); g_clear_object (&temp_info); + g_clear_object (&stored_path); + g_clear_object (&pending_path); ot_clear_checksum (&ret_checksum); return ret; } @@ -1030,30 +1088,13 @@ ostree_repo_prepare_transaction (OstreeRepo *self, { gboolean ret = FALSE; OstreeRepoPrivate *priv = GET_PRIVATE (self); - GString *tmpdir = NULL; - tmpdir = g_string_new (""); - g_return_val_if_fail (priv->in_transaction == FALSE, FALSE); priv->in_transaction = TRUE; - g_clear_object (&priv->transaction_dir); - g_string_append (tmpdir, ot_gfile_get_path_cached (priv->tmp_dir)); - g_string_append_c (tmpdir, '/'); - g_string_append_printf (tmpdir, "trans-%lu.XXXXXX", - (gulong)getpid ()); - if (!mkdtemp (tmpdir->str)) - { - ot_util_set_error_from_errno (error, errno); - goto out; - } - - priv->transaction_dir = ot_gfile_new_for_path (tmpdir->str); ret = TRUE; - out: - if (tmpdir) - g_string_free (tmpdir, TRUE); + /* out: */ return ret; } @@ -1071,11 +1112,10 @@ ostree_repo_commit_transaction (OstreeRepo *self, g_return_val_if_fail (priv->in_transaction == TRUE, FALSE); - g_hash_table_iter_init (&iter, priv->pending_transaction_tmpfiles); + g_hash_table_iter_init (&iter, priv->pending_transaction); while (g_hash_table_iter_next (&iter, &key, &value)) { const char *checksum_and_type = key; - const char *filename = value; const char *type_str; OstreeObjectType objtype; @@ -1087,7 +1127,7 @@ ostree_repo_commit_transaction (OstreeRepo *self, objtype = ostree_object_type_from_string (type_str + 1); g_clear_object (&f); - f = g_file_get_child (priv->transaction_dir, filename); + f = get_pending_object_path (self, checksum, objtype); if (!commit_staged_file (self, f, checksum, objtype, cancellable, error)) goto out; @@ -1095,12 +1135,10 @@ ostree_repo_commit_transaction (OstreeRepo *self, ret = TRUE; out: - (void) rmdir (ot_gfile_get_path_cached (priv->transaction_dir)); priv->in_transaction = FALSE; - g_clear_object (&priv->transaction_dir); g_free (checksum); - g_hash_table_remove_all (priv->pending_transaction_tmpfiles); + g_hash_table_remove_all (priv->pending_transaction); g_clear_object (&f); return ret; } @@ -1112,30 +1150,12 @@ ostree_repo_abort_transaction (OstreeRepo *self, { gboolean ret = FALSE; OstreeRepoPrivate *priv = GET_PRIVATE (self); - GFile *f = NULL; - GHashTableIter iter; - gpointer key, value; - - g_return_val_if_fail (priv->in_transaction == TRUE, FALSE); - - g_hash_table_iter_init (&iter, priv->pending_transaction_tmpfiles); - while (g_hash_table_iter_next (&iter, &key, &value)) - { - const char *filename = value; - - g_clear_object (&f); - f = g_file_get_child (priv->transaction_dir, filename); - - (void) unlink (ot_gfile_get_path_cached (f)); - } + /* For now, let's not delete pending files */ + g_hash_table_remove_all (priv->pending_transaction); priv->in_transaction = FALSE; - (void) rmdir (ot_gfile_get_path_cached (priv->transaction_dir)); - g_clear_object (&priv->transaction_dir); ret = TRUE; - g_hash_table_remove_all (priv->pending_transaction_tmpfiles); - g_clear_object (&f); return ret; } @@ -1179,35 +1199,21 @@ ostree_repo_load_variant (OstreeRepo *self, GError **error) { gboolean ret = FALSE; - OstreeRepoPrivate *priv = GET_PRIVATE (self); GFile *object_path = NULL; GFile *tmpfile = NULL; GVariant *ret_variant = NULL; - char *pending_key = NULL; - const char *pending_tmpfile; g_return_val_if_fail (OSTREE_OBJECT_TYPE_IS_META (expected_type), FALSE); - pending_key = create_checksum_and_objtype (sha256, expected_type); - if ((pending_tmpfile = g_hash_table_lookup (priv->pending_transaction_tmpfiles, pending_key)) != NULL) - { - tmpfile = g_file_get_child (priv->transaction_dir, pending_tmpfile); - if (!ostree_map_metadata_file (tmpfile, expected_type, &ret_variant, error)) - goto out; - } - else - { - object_path = ostree_repo_get_object_path (self, sha256, expected_type); - if (!ostree_map_metadata_file (object_path, expected_type, &ret_variant, error)) - goto out; - } + object_path = ostree_repo_get_object_path (self, sha256, expected_type); + if (!ostree_map_metadata_file (object_path, expected_type, &ret_variant, error)) + goto out; ret = TRUE; ot_transfer_out_value (out_variant, &ret_variant); out: g_clear_object (&object_path); g_clear_object (&tmpfile); - g_free (pending_key); ot_clear_gvariant (&ret_variant); return ret; } diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index d80e12b4..19494464 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -92,12 +92,13 @@ gboolean ostree_repo_abort_transaction (OstreeRepo *self, GCancellable *cancellable, GError **error); -gboolean ostree_repo_has_object (OstreeRepo *self, - OstreeObjectType objtype, - const char *checksum, - gboolean *out_have_object, - GCancellable *cancellable, - GError **error); +gboolean ostree_repo_find_object (OstreeRepo *self, + OstreeObjectType objtype, + const char *checksum, + GFile **out_stored_path, + GFile **out_pending_path, + GCancellable *cancellable, + GError **error); gboolean ostree_repo_stage_object (OstreeRepo *self, OstreeObjectType objtype, diff --git a/src/ostree/ostree-pull.c b/src/ostree/ostree-pull.c index 6b954bac..e17917ef 100644 --- a/src/ostree/ostree-pull.c +++ b/src/ostree/ostree-pull.c @@ -117,119 +117,141 @@ fetch_object (OstreeRepo *repo, SoupURI *baseuri, const char *checksum, OstreeObjectType objtype, - gboolean *did_exist, - GFile **out_file, + GFile **out_temp_path, GError **error) { gboolean ret = FALSE; - GFile *ret_file = NULL; char *objpath = NULL; char *relpath = NULL; SoupURI *obj_uri = NULL; - gboolean exists; + GFile *ret_temp_path = NULL; - g_assert (objtype != OSTREE_OBJECT_TYPE_RAW_FILE); - - if (!ostree_repo_has_object (repo, objtype, checksum, &exists, NULL, error)) + objpath = ostree_get_relative_object_path (checksum, objtype); + obj_uri = soup_uri_copy (baseuri); + relpath = g_build_filename (soup_uri_get_path (obj_uri), objpath, NULL); + soup_uri_set_path (obj_uri, relpath); + + if (!fetch_uri (repo, soup, obj_uri, ostree_object_type_to_string (objtype), &ret_temp_path, error)) goto out; - if (!exists) - { - objpath = ostree_get_relative_object_path (checksum, objtype); - obj_uri = soup_uri_copy (baseuri); - relpath = g_build_filename (soup_uri_get_path (obj_uri), objpath, NULL); - soup_uri_set_path (obj_uri, relpath); - - if (!fetch_uri (repo, soup, obj_uri, ostree_object_type_to_string (objtype), &ret_file, error)) - goto out; - - *did_exist = FALSE; - } - else - *did_exist = TRUE; - ret = TRUE; - ot_transfer_out_value (out_file, &ret_file); + ot_transfer_out_value (out_temp_path, &ret_temp_path); out: if (obj_uri) soup_uri_free (obj_uri); - g_clear_object (&ret_file); + g_clear_object (&ret_temp_path); g_free (objpath); g_free (relpath); return ret; } static gboolean -store_object (OstreeRepo *repo, - SoupSession *soup, - SoupURI *baseuri, - const char *checksum, - OstreeObjectType objtype, - gboolean *did_exist, - GError **error) +fetch_and_store_object (OstreeRepo *repo, + SoupSession *soup, + SoupURI *baseuri, + const char *checksum, + OstreeObjectType objtype, + gboolean *out_is_pending, + GVariant **out_metadata, + GError **error) { gboolean ret = FALSE; - GFile *filename = NULL; GFileInfo *file_info = NULL; GInputStream *input = NULL; + GFile *stored_path = NULL; + GFile *pending_path = NULL; + GFile *temp_path = NULL; + GVariant *ret_metadata = NULL; + gboolean ret_is_pending; + + g_assert (objtype != OSTREE_OBJECT_TYPE_RAW_FILE); - if (!fetch_object (repo, soup, baseuri, checksum, objtype, did_exist, &filename, error)) + if (!ostree_repo_find_object (repo, objtype, checksum, + &stored_path, &pending_path, NULL, error)) goto out; + + if (!(stored_path || pending_path)) + { + if (!fetch_object (repo, soup, baseuri, checksum, objtype, &temp_path, error)) + goto out; + } - if (!*did_exist) + if (temp_path) { - file_info = g_file_query_info (filename, OSTREE_GIO_FAST_QUERYINFO, + file_info = g_file_query_info (temp_path, OSTREE_GIO_FAST_QUERYINFO, G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, NULL, error); if (!file_info) goto out; - input = (GInputStream*)g_file_read (filename, NULL, error); + input = (GInputStream*)g_file_read (temp_path, NULL, error); if (!input) goto out; + } + if (pending_path || temp_path) + { if (!ostree_repo_stage_object (repo, objtype, checksum, file_info, NULL, input, NULL, error)) goto out; + + log_verbose ("Staged object: %s.%s", checksum, ostree_object_type_to_string (objtype)); + + ret_is_pending = TRUE; + if (out_metadata) + { + if (!ostree_map_metadata_file (pending_path ? pending_path : temp_path, objtype, &ret_metadata, error)) + goto out; + } + } + else + { + ret_is_pending = FALSE; } ret = TRUE; + if (out_is_pending) + *out_is_pending = ret_is_pending; + ot_transfer_out_value (out_metadata, &ret_metadata); out: - if (filename) - (void) unlink (ot_gfile_get_path_cached (filename)); + if (temp_path) + (void) unlink (ot_gfile_get_path_cached (temp_path)); + ot_clear_gvariant (&ret_metadata); + g_clear_object (&temp_path); g_clear_object (&file_info); g_clear_object (&input); + g_clear_object (&stored_path); + g_clear_object (&pending_path); return ret; } static gboolean -store_tree_recurse (OstreeRepo *repo, - SoupSession *soup, - SoupURI *base_uri, - const char *rev, - GError **error) +fetch_and_store_tree_recurse (OstreeRepo *repo, + SoupSession *soup, + SoupURI *base_uri, + const char *rev, + GError **error) { gboolean ret = FALSE; GVariant *tree = NULL; GVariant *files_variant = NULL; GVariant *dirs_variant = NULL; - gboolean did_exist; + gboolean is_pending; int i, n; GVariant *archive_metadata = NULL; GFileInfo *archive_file_info = NULL; GVariant *archive_xattrs = NULL; - GFile *meta_file = NULL; - GFile *content_file = NULL; + GFile *meta_temp_path = NULL; + GFile *content_temp_path = NULL; + GFile *stored_path = NULL; + GFile *pending_path = NULL; GInputStream *input = NULL; - if (!store_object (repo, soup, base_uri, rev, OSTREE_OBJECT_TYPE_DIR_TREE, &did_exist, error)) + if (!fetch_and_store_object (repo, soup, base_uri, rev, OSTREE_OBJECT_TYPE_DIR_TREE, &is_pending, &tree, error)) goto out; - if (did_exist) + if (!is_pending) log_verbose ("Already have tree %s", rev); else { - if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_DIR_TREE, rev, &tree, error)) - goto out; - /* PARSE OSTREE_SERIALIZED_TREE_VARIANT */ files_variant = g_variant_get_child_value (tree, 2); dirs_variant = g_variant_get_child_value (tree, 3); @@ -247,43 +269,79 @@ store_tree_recurse (OstreeRepo *repo, if (!ostree_validate_checksum_string (checksum, error)) goto out; - g_clear_object (&meta_file); - - if (!fetch_object (repo, soup, base_uri, checksum, - OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META, - &did_exist, - &meta_file, - error)) - goto out; - - if (!ostree_map_metadata_file (meta_file, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META, - &archive_metadata, error)) - goto out; - - if (!ostree_parse_archived_file_meta (archive_metadata, &archive_file_info, &archive_xattrs, error)) - goto out; + g_clear_object (&stored_path); + g_clear_object (&pending_path); + /* If we're fetching from an archive into a bare repository, we need + * to explicitly check for raw file types locally. + */ + if (ostree_repo_get_mode (repo) == OSTREE_REPO_MODE_BARE) + { + if (!ostree_repo_find_object (repo, OSTREE_OBJECT_TYPE_RAW_FILE, checksum, + &stored_path, &pending_path, NULL, error)) + goto out; + } + else + { + if (!ostree_repo_find_object (repo, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT, checksum, + &stored_path, &pending_path, NULL, error)) + goto out; + } g_clear_object (&input); - g_clear_object (&content_file); - if (g_file_info_get_file_type (archive_file_info) == G_FILE_TYPE_REGULAR) + g_clear_object (&archive_file_info); + ot_clear_gvariant (&archive_xattrs); + if (!(stored_path || pending_path)) { + g_clear_object (&meta_temp_path); if (!fetch_object (repo, soup, base_uri, checksum, - OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT, - &did_exist, - &content_file, + OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META, + &meta_temp_path, error)) goto out; - - input = (GInputStream*)g_file_read (content_file, NULL, error); - if (!input) + + if (!ostree_map_metadata_file (meta_temp_path, + OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META, + &archive_metadata, error)) + goto out; + + if (!ostree_parse_archived_file_meta (archive_metadata, &archive_file_info, &archive_xattrs, error)) goto out; + + if (g_file_info_get_file_type (archive_file_info) == G_FILE_TYPE_REGULAR) + { + if (!fetch_object (repo, soup, base_uri, checksum, + OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT, + &content_temp_path, + error)) + goto out; + + input = (GInputStream*)g_file_read (content_temp_path, NULL, error); + if (!input) + goto out; + } } - if (!ostree_repo_stage_object (repo, OSTREE_OBJECT_TYPE_RAW_FILE, - checksum, - archive_file_info, archive_xattrs, input, - NULL, error)) - goto out; + if (!stored_path) + { + log_verbose ("Staged file object: %s", checksum); + + if (!ostree_repo_stage_object (repo, OSTREE_OBJECT_TYPE_RAW_FILE, + checksum, + archive_file_info, archive_xattrs, input, + NULL, error)) + goto out; + } + + if (meta_temp_path) + { + (void) unlink (ot_gfile_get_path_cached (meta_temp_path)); + g_clear_object (&meta_temp_path); + } + if (content_temp_path) + { + (void) unlink (ot_gfile_get_path_cached (content_temp_path)); + g_clear_object (&content_temp_path); + } } n = g_variant_n_children (dirs_variant); @@ -303,10 +361,10 @@ store_tree_recurse (OstreeRepo *repo, if (!ostree_validate_checksum_string (meta_checksum, error)) goto out; - if (!store_object (repo, soup, base_uri, meta_checksum, OSTREE_OBJECT_TYPE_DIR_META, &did_exist, error)) + if (!fetch_and_store_object (repo, soup, base_uri, meta_checksum, OSTREE_OBJECT_TYPE_DIR_META, NULL, NULL, error)) goto out; - if (!store_tree_recurse (repo, soup, base_uri, tree_checksum, error)) + if (!fetch_and_store_tree_recurse (repo, soup, base_uri, tree_checksum, error)) goto out; } } @@ -320,40 +378,49 @@ store_tree_recurse (OstreeRepo *repo, ot_clear_gvariant (&archive_xattrs); g_clear_object (&archive_file_info); g_clear_object (&input); + g_clear_object (&stored_path); + g_clear_object (&pending_path); + if (content_temp_path) + { + (void) unlink (ot_gfile_get_path_cached (content_temp_path)); + g_clear_object (&content_temp_path); + } + if (meta_temp_path) + { + (void) unlink (ot_gfile_get_path_cached (meta_temp_path)); + g_clear_object (&meta_temp_path); + } return ret; } static gboolean -store_commit_recurse (OstreeRepo *repo, - SoupSession *soup, - SoupURI *base_uri, - const char *rev, - GError **error) +fetch_and_store_commit_recurse (OstreeRepo *repo, + SoupSession *soup, + SoupURI *base_uri, + const char *rev, + GError **error) { gboolean ret = FALSE; GVariant *commit = NULL; const char *tree_contents_checksum; const char *tree_meta_checksum; - gboolean did_exist; + gboolean is_pending; - if (!store_object (repo, soup, base_uri, rev, OSTREE_OBJECT_TYPE_COMMIT, &did_exist, error)) + if (!fetch_and_store_object (repo, soup, base_uri, rev, OSTREE_OBJECT_TYPE_COMMIT, &is_pending, &commit, error)) goto out; - if (did_exist) + if (!is_pending) log_verbose ("Already have commit %s", rev); else { - if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT, rev, &commit, error)) - goto out; - /* PARSE OSTREE_SERIALIZED_COMMIT_VARIANT */ g_variant_get_child (commit, 6, "&s", &tree_contents_checksum); g_variant_get_child (commit, 7, "&s", &tree_meta_checksum); - if (!store_object (repo, soup, base_uri, tree_meta_checksum, OSTREE_OBJECT_TYPE_DIR_META, &did_exist, error)) + if (!fetch_and_store_object (repo, soup, base_uri, tree_meta_checksum, OSTREE_OBJECT_TYPE_DIR_META, NULL, NULL, error)) goto out; - if (!store_tree_recurse (repo, soup, base_uri, tree_contents_checksum, error)) + if (!fetch_and_store_tree_recurse (repo, soup, base_uri, tree_contents_checksum, error)) goto out; } @@ -450,7 +517,7 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error) if (!ostree_repo_prepare_transaction (repo, NULL, error)) goto out; - if (!store_commit_recurse (repo, soup, base_uri, rev, error)) + if (!fetch_and_store_commit_recurse (repo, soup, base_uri, rev, error)) goto out; if (!ostree_repo_commit_transaction (repo, NULL, error)) -- 2.30.2